{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "import random\n",
    "import pymongo\n",
    "\n",
    "client = pymongo.MongoClient(host = \"127.0.0.1\",port=27017)\n",
    "db = client[\"spider\"]\n",
    "collection = db[\"lagou\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cookie():\n",
    "    headers = {\n",
    "    'Connection': 'keep-alive',\n",
    "    'Cache-Control': 'max-age=0',\n",
    "    'Upgrade-Insecure-Requests': '1',\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36',\n",
    "    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',\n",
    "    'Accept-Encoding': 'gzip, deflate, br',\n",
    "    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',\n",
    "}\n",
    "    response = requests.get(\n",
    "        'https://www.lagou.com/jobs/list_?city=%E4%B8%8A%E6%B5%B7&cl=false&fromSearch=true&labelWords=&suginput=',\n",
    "        headers=headers)  # 请求原网页\n",
    "    r = requests.utils.dict_from_cookiejar(response.cookies)  # 获取cookies\n",
    "\n",
    "    cookies = {\n",
    "        'X_MIDDLE_TOKEN': '797bc148d133274a162ba797a6875817',\n",
    "        'JSESSIONID': 'ABAAABAAAIAACBI03F33A375F98E05C5108D4D742A34114',\n",
    "        '_ga': 'GA1.2.1912257997.1548059451',\n",
    "        '_gat': '1',\n",
    "        'Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6': '1548059451',\n",
    "        'user_trace_token': '20190121163050-dbd72da2-1d56-11e9-8927-525400f775ce',\n",
    "        'LGSID': '20190121163050-dbd72f67-1d56-11e9-8927-525400f775ce',\n",
    "        'PRE_UTM': '',\n",
    "        'PRE_HOST': '',\n",
    "        'PRE_SITE': '',\n",
    "        'PRE_LAND': 'https%3A%2F%2Fwww.lagou.com%2F%3F_from_mid%3D1',\n",
    "        'LGUID': '20190121163050-dbd73128-1d56-11e9-8927-525400f775ce',\n",
    "        '_gid': 'GA1.2.1194828713.1548059451',\n",
    "        'index_location_city': '%E5%85%A8%E5%9B%BD',\n",
    "        'TG-TRACK-CODE': 'index_hotjob',\n",
    "        'LGRID': '20190121163142-fb0cc9c0-1d56-11e9-8928-525400f775ce',\n",
    "        'Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6': '1548059503',\n",
    "        'SEARCH_ID': '86ed37f5d8da417dafb53aa25cd6fbc0',\n",
    "    }\n",
    "    cookies.update(r)  # 更新接口的cookies\n",
    "    return cookies\n",
    "\n",
    "def crawl(city = \"\", pn = 1, cookies = None):\n",
    "    headers = {\n",
    "        'Origin': 'https://www.lagou.com',\n",
    "        'X-Anit-Forge-Code': '0',\n",
    "        'Accept-Encoding': 'gzip, deflate, br',\n",
    "        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',\n",
    "        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36',\n",
    "        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',\n",
    "        'Accept': 'application/json, text/javascript, */*; q=0.01',\n",
    "        'Referer': 'https://www.lagou.com/jobs/list_java?px=new&city=%E4%B8%8A%E6%B5%B7',\n",
    "        'X-Requested-With': 'XMLHttpRequest',\n",
    "        'Connection': 'keep-alive',\n",
    "        'X-Anit-Forge-Token': 'None',\n",
    "    }\n",
    "\n",
    "    params = (\n",
    "        ('px', 'default'),\n",
    "        ('city',city),\n",
    "        ('needAddtionalResult', 'false'),\n",
    "    )\n",
    "\n",
    "    data = {\"first\":\"true\",\n",
    "            'kd': '数据分析',\n",
    "            'pn': pn}\n",
    "    if pn>1:\n",
    "        data[\"first\"] = \"false\"\n",
    "    response = requests.post('https://www.lagou.com/jobs/positionAjax.json', headers=headers, params=params,\n",
    "                             cookies=cookies, data=data)  # 请求接口\n",
    "    return response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "city_list = [\"北京\",\"上海\",\"深圳\",\"广州\",\"杭州\",\"成都\",\"南京\",\"武汉\",\"西安\",\"厦门\",\"长沙\",\"苏州\",\"天津\"]\n",
    "\n",
    "for city in city_list:\n",
    "    print(\"*\"*60)\n",
    "    print(\"{city} start\".format(city=city))\n",
    "    for i in range(1,31):\n",
    "        if (i-1)%5==0:\n",
    "            cookies = get_cookie()\n",
    "        time.sleep(random.random()+random.randint(1,2))\n",
    "        response_json = crawl(city=city,pn=i,cookies=cookies)\n",
    "        try:\n",
    "            position_list = response_json[\"content\"][ 'positionResult'][\"result\"]\n",
    "        except:\n",
    "            print(response_json)\n",
    "        if len(position_list)<1: \n",
    "            print(\"{city} start\".format(city=city))\n",
    "            print(\"*\"*60)\n",
    "            break\n",
    "        collection.insert_many(position_list)\n",
    "        print(cookies)\n",
    "    print(\"{city} end\".format(city=city))\n",
    "    print(\"*\"*60)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
